
package WebExample;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;

public class Crawler extends WebCrawler {

    // file name for properties file to load
    final static String PROPCONFIG = "src\\main\\java\\WebExample\\crawlerLogic.properties";
    final static String RESTRICT = "src\\main\\java\\WebExample\\sites.properties"; //addded by Sanjiv Kawa

    /*
     * Boolean to determine if website should be crawled or not
     */
    @Override
    public boolean shouldVisit(WebURL url) {

        Properties myProps = new Properties();
        FileInputStream propInputStream;
        try {
            propInputStream = new FileInputStream(PROPCONFIG);
            myProps.load(propInputStream);
        } catch (FileNotFoundException e1) {
            e1.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }


        //loads patterns from .properties
        Pattern FILTERS = null;
        Pattern FILTERS1 = null;
        if (!(myProps.getProperty("regexUrlFileType") == null)) {
            FILTERS = Pattern.compile(myProps.getProperty("regexUrlFileType"));

        }
        if (!(myProps.getProperty("regexUrlWords") == null)) {
            FILTERS1 = Pattern.compile(myProps.getProperty("regexUrlWords"));
        }
        String href = url.getURL().toLowerCase();
        //boolean flagged = true;




       /* try {
            //flagged = MySQLjdbc.checkSiteFlagged(href);
        } catch (SQLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }*/

        // returns true if does not match patters
        return !FILTERS.matcher(href).matches() && !FILTERS1.matcher(href).find();
    }


    /*
     * Vists page and gets information, checks content against filters
     */
    @Override
    public void visit(Page page) {

        int docid = page.getWebURL().getDocid();
        String url = page.getWebURL().getURL();
        String domain = page.getWebURL().getDomain();
        String path = page.getWebURL().getPath();
        String subDomain = page.getWebURL().getSubDomain();
        String parentUrl = page.getWebURL().getParentUrl();


        Properties restrictToURL = new Properties(); //added by sanjiv kawa
        FileInputStream propInStream = null; //added by sanjiv kawa

        try //added by sanjiv kawa
        {
            propInStream = new FileInputStream(RESTRICT);
        } catch (FileNotFoundException ex) {
            Logger.getLogger(Crawler.class.getName()).log(Level.SEVERE, null, ex);
        }
        try //added by sanjiv kawa
        {
            restrictToURL.load(propInStream);
        } catch (IOException ex) {
            Logger.getLogger(Crawler.class.getName()).log(Level.SEVERE, null, ex);
        }

        String restrictedToURL1 = restrictToURL.getProperty("restrict1"); //added by sanjiv kawa
        String restrictedToURL2 = restrictToURL.getProperty("restrict2"); //added by sanjiv kawa

        boolean patternString1 = org.apache.commons.lang3.StringUtils.contains(url, restrictedToURL1); //added by sanjiv kawa
        boolean patternString2 = org.apache.commons.lang3.StringUtils.contains(url, restrictedToURL2); //added by sanjiv kawa


        if (patternString1 == true || patternString2 == true) //added by sanjiv kawa - restricts urls to fit /thread/ or /threads/ format
        {

            System.out.println("Docid: " + docid);
            System.out.println("URL: " + url);
            System.out.println("Domain: '" + domain + "'");
            System.out.println("Sub-domain: '" + subDomain + "'");
            System.out.println("Path: '" + path + "'");
            System.out.println("Parent page: " + parentUrl);


            try {
                DownloadPage.writeURLtoFile(page);
            } catch (IOException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
            }


            /*try {

				//filterHTML.filterThreadPosts(page);
				
				
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (ClassNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} /*catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}*/


            System.out.println("======================================");
        }
    }
    
    
    /*
     * Before exit of thread email is sent 
     */
    /*
    public void onBeforeExit() {
    	try {
			sendEmail.sendmail();
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		} catch (SQLException e) {
			e.printStackTrace();
		}
		
    
    }
       */
}